
#ifndef GH_PTX_H_INCLUDED
#define GH_PTX_H_INCLUDED

#ifndef NL
    #define NL "\n"
#endif

// generic vec2 operations
#define ptx_op_v2(op, d, v)     \
    NL "    "op" "d".x, "v".x;" \
    NL "    "op" "d".y, "v".y;"

#define ptx_op_v2s(op, d, v, s)      \
    NL "    "op" "d".x, "v".x, "s";" \
    NL "    "op" "d".y, "v".y, "s";"

#define ptx_op_v2v2(op, d, v0, v1)       \
    NL "    "op" "d".x, "v0".x, "v1".x;" \
    NL "    "op" "d".y, "v0".y, "v1".y;"

#define ptx_op_v2sv2(op, d, v0, s, v1)        \
    NL "    "op" "d".x, "v0".x, "s", "v1".x;" \
    NL "    "op" "d".y, "v0".y, "s", "v1".y;"

#define ptx_op_v2v2s(op, d, v0, v1, s)        \
    NL "    "op" "d".x, "v0".x, "v1".x, "s";" \
    NL "    "op" "d".y, "v0".y, "v1".y, "s";"

#define ptx_op_v2ss(op, d, v, s0, s1)       \
    NL "    "op" "d".x, "v".x, "s0", "s1";" \
    NL "    "op" "d".y, "v".y, "s0", "s1";"

#define ptx_op_v2v2v2(op, d, v0, v1, v2)         \
    NL "    "op" "d".x, "v0".x, "v1".x, "v2".x;" \
    NL "    "op" "d".y, "v0".y, "v1".y, "v2".y;"

// generic vec3 operations
#define ptx_op_v3(op, d, v)     \
    NL "    "op" "d".x, "v".x;" \
    NL "    "op" "d".y, "v".y;" \
    NL "    "op" "d".z, "v".z;"

#define ptx_op_v3s(op, d, v, s)      \
    NL "    "op" "d".x, "v".x, "s";" \
    NL "    "op" "d".y, "v".y, "s";" \
    NL "    "op" "d".z, "v".z, "s";"

#define ptx_op_v3v3(op, d, v0, v1)       \
    NL "    "op" "d".x, "v0".x, "v1".x;" \
    NL "    "op" "d".y, "v0".y, "v1".y;" \
    NL "    "op" "d".z, "v0".z, "v1".z;"

#define ptx_op_v3sv3(op, d, v0, s, v1)        \
    NL "    "op" "d".x, "v0".x, "s", "v1".x;" \
    NL "    "op" "d".y, "v0".y, "s", "v1".y;" \
    NL "    "op" "d".z, "v0".z, "s", "v1".z;"

#define ptx_op_v3v3s(op, d, v0, v1, s)        \
    NL "    "op" "d".x, "v0".x, "v1".x, "s";" \
    NL "    "op" "d".y, "v0".y, "v1".y, "s";" \
    NL "    "op" "d".z, "v0".z, "v1".z, "s";" \

#define ptx_op_v3ss(op, d, v, s0, s1)       \
    NL "    "op" "d".x, "v".x, "s0", "s1";" \
    NL "    "op" "d".y, "v".y, "s0", "s1";" \
    NL "    "op" "d".z, "v".z, "s0", "s1";"

#define ptx_op_v3v3v3(op, d, v0, v1, v2)         \
    NL "    "op" "d".x, "v0".x, "v1".x, "v2".x;" \
    NL "    "op" "d".y, "v0".y, "v1".y, "v2".y;" \
    NL "    "op" "d".z, "v0".z, "v1".z, "v2".z;"

// generic vec4 operations
#define ptx_op_v4(op, d, v)     \
    NL "    "op" "d".x, "v".x;" \
    NL "    "op" "d".y, "v".y;" \
    NL "    "op" "d".z, "v".z;" \
    NL "    "op" "d".w, "v".w;"

#define ptx_op_v4s(op, d, v, s)      \
    NL "    "op" "d".x, "v".x, "s";" \
    NL "    "op" "d".y, "v".y, "s";" \
    NL "    "op" "d".z, "v".z, "s";" \
    NL "    "op" "d".w, "v".w, "s";"

#define ptx_op_v4v4(op, d, v0, v1)       \
    NL "    "op" "d".x, "v0".x, "v1".x;" \
    NL "    "op" "d".y, "v0".y, "v1".y;" \
    NL "    "op" "d".z, "v0".z, "v1".z;" \
    NL "    "op" "d".w, "v0".w, "v1".w;"

#define ptx_op_v4sv4(op, d, v0, s, v1)        \
    NL "    "op" "d".x, "v0".x, "s", "v1".x;" \
    NL "    "op" "d".y, "v0".y, "s", "v1".y;" \
    NL "    "op" "d".z, "v0".z, "s", "v1".z;" \
    NL "    "op" "d".w, "v0".w, "s", "v1".w;"

#define ptx_op_v4v4s(op, d, v0, v1, s)        \
    NL "    "op" "d".x, "v0".x, "v1".x, "s";" \
    NL "    "op" "d".y, "v0".y, "v1".y, "s";" \
    NL "    "op" "d".z, "v0".z, "v1".z, "s";" \
    NL "    "op" "d".w, "v0".w, "v1".w, "s";"

#define ptx_op_v4ss(op, d, v, s0, s1)       \
    NL "    "op" "d".x, "v".x, "s0", "s1";" \
    NL "    "op" "d".y, "v".y, "s0", "s1";" \
    NL "    "op" "d".z, "v".z, "s0", "s1";" \
    NL "    "op" "d".w, "v".w, "s0", "s1";"

#define ptx_op_v4v4v4(op, d, v0, v1, v2)         \
    NL "    "op" "d".x, "v0".x, "v1".x, "v2".x;" \
    NL "    "op" "d".y, "v0".y, "v1".y, "v2".y;" \
    NL "    "op" "d".z, "v0".z, "v1".z, "v2".z;" \
    NL "    "op" "d".w, "v0".w, "v1".w, "v2".w;"

// add_vX_f32
#define add_v2_f32(d, v0, v1) ptx_op_v2v2("add.f32", d, v0, v1)
#define add_v3_f32(d, v0, v1) ptx_op_v3v3("add.f32", d, v0, v1)
#define add_v4_f32(d, v0, v1) ptx_op_v4v4("add.f32", d, v0, v1)

// sub_vX_f32
#define sub_v2_f32(d, v0, v1) ptx_op_v2v2("sub.f32", d, v0, v1)
#define sub_v3_f32(d, v0, v1) ptx_op_v3v3("sub.f32", d, v0, v1)
#define sub_v4_f32(d, v0, v1) ptx_op_v4v4("sub.f32", d, v0, v1)

// mul_vX_f32
#define mul_v2_f32(d, v0, v1) ptx_op_v2v2("mul.f32", d, v0, v1)
#define mul_v3_f32(d, v0, v1) ptx_op_v3v3("mul.f32", d, v0, v1)
#define mul_v4_f32(d, v0, v1) ptx_op_v4v4("mul.f32", d, v0, v1)

// div_approx_vX_f32
#define div_approx_v2_f32(d, v0, v1) ptx_op_v2v2("div.approx.f32", d, v0, v1)
#define div_approx_v3_f32(d, v0, v1) ptx_op_v3v3("div.approx.f32", d, v0, v1)
#define div_approx_v4_f32(d, v0, v1) ptx_op_v4v4("div.approx.f32", d, v0, v1)

// add_vXs_f32
#define add_v2s_f32(d, v, s) ptx_op_v2s("add.f32", d, v, s)
#define add_v3s_f32(d, v, s) ptx_op_v3s("add.f32", d, v, s)
#define add_v4s_f32(d, v, s) ptx_op_v4s("add.f32", d, v, s)

// mul_vXs_f32
#define mul_v2s_f32(d, v, s) ptx_op_v2s("mul.f32", d, v, s)
#define mul_v3s_f32(d, v, s) ptx_op_v3s("mul.f32", d, v, s)
#define mul_v4s_f32(d, v, s) ptx_op_v4s("mul.f32", d, v, s)

// fma_vXsvX_f32
#define fma_v2sv2_f32(d, v0, s, v1) ptx_op_v2sv2("fma.rn.f32", d, v0, s, v1)
#define fma_v3sv3_f32(d, v0, s, v1) ptx_op_v3sv3("fma.rn.f32", d, v0, s, v1)
#define fma_v4sv4_f32(d, v0, s, v1) ptx_op_v4sv4("fma.rn.f32", d, v0, s, v1)

// fma_vXvXs_f32
#define fma_v2v2s_f32(d, v0, v1, s) ptx_op_v2v2s("fma.rn.f32", d, v0, v1, s)
#define fma_v3v3s_f32(d, v0, v1, s) ptx_op_v3v3s("fma.rn.f32", d, v0, v1, s)
#define fma_v4v4s_f32(d, v0, v1, s) ptx_op_v4v4s("fma.rn.f32", d, v0, v1, s)

// fma_vXss_f32
#define fma_v2ss_f32(d, v, s0, s1) ptx_op_v2ss("fma.rn.f32", d, v, s0, s1)
#define fma_v3ss_f32(d, v, s0, s1) ptx_op_v3ss("fma.rn.f32", d, v, s0, s1)
#define fma_v4ss_f32(d, v, s0, s1) ptx_op_v4ss("fma.rn.f32", d, v, s0, s1)

// fma_vX_f32
#define fma_v2_f32(d, v0, v1, v2) ptx_op_v2v2v2("fma.rn.f32", d, v0, v1, v2)
#define fma_v3_f32(d, v0, v1, v2) ptx_op_v3v3v3("fma.rn.f32", d, v0, v1, v2)
#define fma_v4_f32(d, v0, v1, v2) ptx_op_v4v4v4("fma.rn.f32", d, v0, v1, v2)

// generic dot operations
#define ptx_dot_v2(type, d, a, b)                    \
    NL "    mul"type"       "d", "a".x, "b".x;"      \
    NL "    fma.rn"type"    "d", "a".y, "b".y, "d";"

#define ptx_dot_v3(type, d, a, b)                    \
    NL "    mul"type"       "d", "a".x, "b".x;"      \
    NL "    fma.rn"type"    "d", "a".y, "b".y, "d";" \
    NL "    fma.rn"type"    "d", "a".z, "b".z, "d";"

#define ptx_dot_v4(type, d, a, b)                    \
    NL "    mul"type"       "d", "a".x, "b".x;"      \
    NL "    fma.rn"type"    "d", "a".y, "b".y, "d";" \
    NL "    fma.rn"type"    "d", "a".z, "b".z, "d";" \
    NL "    fma.rn"type"    "d", "a".w, "b".w, "d";"

// dot_vX_f32
#define dot_v2_f32(d, a, b) ptx_dot_v2(".f32", d, a, b)
#define dot_v3_f32(d, a, b) ptx_dot_v3(".f32", d, a, b)
#define dot_v4_f32(d, a, b) ptx_dot_v4(".f32", d, a, b)

// generic saturated dot operations
#define ptx_dot_sat_v2(type, d, a, b)                    \
    NL "    mul"type"           "d", "a".x, "b".x;"      \
    NL "    fma.rn.sat"type"    "d", "a".y, "b".y, "d";"

#define ptx_dot_sat_v3(type, d, a, b)                    \
    NL "    mul"type"           "d", "a".x, "b".x;"      \
    NL "    fma.rn"type"        "d", "a".y, "b".y, "d";" \
    NL "    fma.rn.sat"type"    "d", "a".z, "b".z, "d";"

#define ptx_dot_sat_v4(type, d, a, b)                    \
    NL "    mul"type"           "d", "a".x, "b".x;"      \
    NL "    fma.rn"type"        "d", "a".y, "b".y, "d";" \
    NL "    fma.rn"type"        "d", "a".z, "b".z, "d";" \
    NL "    fma.rn.sat"type"    "d", "a".w, "b".w, "d";"

// dot_sat_vX_f32
#define dot_sat_v2_f32(d, a, b) ptx_dot_sat_v2(".f32", d, a, b)
#define dot_sat_v3_f32(d, a, b) ptx_dot_sat_v3(".f32", d, a, b)
#define dot_sat_v4_f32(d, a, b) ptx_dot_sat_v4(".f32", d, a, b)

// generic matrix * vector operations
#define ptx_mul_m2v2(type, d, m, v)                  \
    NL "    /* "d" = "m" * "v"; */ {"                \
    NL "    .reg .v2 "type" t;"                      \
    NL "    mov.v2"type"    t, "v";"                 \
            ptx_dot_v2      (type, d".x", m"0", "t") \
            ptx_dot_v2      (type, d".y", m"1", "t") \
    NL "    }"

#define ptx_mul_m3v3(type, d, m, v)                  \
    NL "    /* "d" = "m" * "v"; */ {"                \
    NL "    .reg .v4 "type" t;"                      \
    NL "    mov.v4"type"    t, "v";"                 \
            ptx_dot_v3      (type, d".x", m"0", "t") \
            ptx_dot_v3      (type, d".y", m"1", "t") \
            ptx_dot_v3      (type, d".z", m"2", "t") \
    NL "    }"

#define ptx_mul_m4v4(type, d, m, v)                  \
    NL "    /* "d" = "m" * "v"; */ {"                \
    NL "    .reg .v4 "type" t;"                      \
    NL "    mov.v4"type"    t, "v";"                 \
            ptx_dot_v4      (type, d".x", m"0", "t") \
            ptx_dot_v4      (type, d".y", m"1", "t") \
            ptx_dot_v4      (type, d".z", m"2", "t") \
            ptx_dot_v4      (type, d".w", m"3", "t") \
    NL "    }"

// mul_mXvX_f32
#define mul_m2v2_f32(d, m, v) ptx_mul_m2v2(".f32", d, m, v)
#define mul_m3v3_f32(d, m, v) ptx_mul_m3v3(".f32", d, m, v)
#define mul_m4v4_f32(d, m, v) ptx_mul_m4v4(".f32", d, m, v)

// generic matrix + scalar operations
#define ptx_add_m2s(type, d, m, s)       \
    ptx_op_v2s("add"type, d"0", m"0", s) \
    ptx_op_v2s("add"type, d"1", m"1", s)

#define ptx_add_m3s(type, d, m, s)       \
    ptx_op_v3s("add"type, d"0", m"0", s) \
    ptx_op_v3s("add"type, d"1", m"1", s) \
    ptx_op_v3s("add"type, d"2", m"2", s)

#define ptx_add_m4s(type, d, m, s)       \
    ptx_op_v4s("add"type, d"0", m"0", s) \
    ptx_op_v4s("add"type, d"1", m"1", s) \
    ptx_op_v4s("add"type, d"2", m"2", s) \
    ptx_op_v4s("add"type, d"3", m"3", s)

// add_mXs_f32
#define add_m2s_f32(d, m, s) ptx_add_m2s(".f32", d, m, s)
#define add_m3s_f32(d, m, s) ptx_add_m3s(".f32", d, m, s)
#define add_m4s_f32(d, m, s) ptx_add_m4s(".f32", d, m, s)

// generic matrix * scalar operations
#define ptx_mul_m2s(type, d, m, s)       \
    ptx_op_v2s("mul"type, d"0", m"0", s) \
    ptx_op_v2s("mul"type, d"1", m"1", s)

#define ptx_mul_m3s(type, d, m, s)       \
    ptx_op_v3s("mul"type, d"0", m"0", s) \
    ptx_op_v3s("mul"type, d"1", m"1", s) \
    ptx_op_v3s("mul"type, d"2", m"2", s)

#define ptx_mul_m4s(type, d, m, s)       \
    ptx_op_v4s("mul"type, d"0", m"0", s) \
    ptx_op_v4s("mul"type, d"1", m"1", s) \
    ptx_op_v4s("mul"type, d"2", m"2", s) \
    ptx_op_v4s("mul"type, d"3", m"3", s)

// mul_mXs_f32
#define mul_m2s_f32(d, m, s) ptx_mul_m2s(".f32", d, m, s)
#define mul_m3s_f32(d, m, s) ptx_mul_m3s(".f32", d, m, s)
#define mul_m4s_f32(d, m, s) ptx_mul_m4s(".f32", d, m, s)

// generic matrix * scalar + scalar operations
#define ptx_fma_m2ss(type, d, m, s0, s1)          \
    ptx_op_v2ss("fma.rn"type, d"0", m"0", s0, s1) \
    ptx_op_v2ss("fma.rn"type, d"1", m"1", s0, s1)

#define ptx_fma_m3ss(type, d, m, s0, s1)          \
    ptx_op_v3ss("fma.rn"type, d"0", m"0", s0, s1) \
    ptx_op_v3ss("fma.rn"type, d"1", m"1", s0, s1) \
    ptx_op_v3ss("fma.rn"type, d"1", m"1", s0, s1)

#define ptx_fma_m4ss(type, d, m, s0, s1)          \
    ptx_op_v4ss("fma.rn"type, d"0", m"0", s0, s1) \
    ptx_op_v4ss("fma.rn"type, d"1", m"1", s0, s1) \
    ptx_op_v4ss("fma.rn"type, d"1", m"1", s0, s1) \
    ptx_op_v4ss("fma.rn"type, d"1", m"1", s0, s1)

// fma_mXss_f32
#define fma_m2ss_f32(d, m, s0, s1) ptx_fma_m2ss(".f32", d, m, s0, s1)
#define fma_m3ss_f32(d, m, s0, s1) ptx_fma_m3ss(".f32", d, m, s0, s1)
#define fma_m4ss_f32(d, m, s0, s1) ptx_fma_m4ss(".f32", d, m, s0, s1)

// generic matrix + matrix operations
#define ptx_add_m2m2(type, d, ma, mb)          \
    ptx_op_v2v2("add"type, d"0", ma"0", mb"0") \
    ptx_op_v2v2("add"type, d"1", ma"1", mb"1")

#define ptx_add_m3m3(type, d, ma, mb)          \
    ptx_op_v3v3("add"type, d"0", ma"0", mb"0") \
    ptx_op_v3v3("add"type, d"1", ma"1", mb"1") \
    ptx_op_v3v3("add"type, d"2", ma"2", mb"2")

#define ptx_add_m4m4(type, d, ma, mb)          \
    ptx_op_v4v4("add"type, d"0", ma"0", mb"0") \
    ptx_op_v4v4("add"type, d"1", ma"1", mb"1") \
    ptx_op_v4v4("add"type, d"2", ma"2", mb"2") \
    ptx_op_v4v4("add"type, d"3", ma"3", mb"3")

// add_mX_f32
#define add_m2_f32(d, ma, mb) ptx_add_m2m2(".f32", d, ma, mb)
#define add_m3_f32(d, ma, mb) ptx_add_m3m3(".f32", d, ma, mb)
#define add_m4_f32(d, ma, mb) ptx_add_m4m4(".f32", d, ma, mb)

// generic matrix * matrix operations
#define ptx_mul_m2m2(type, dm, ma, mb)                     \
    NL "    /* "dm" = "ma" * "mb"; */ {"                   \
    NL "    .reg .v2 "type" t<2>;"                         \
    NL "    mul"type"       t0.x, "ma"0.x, "mb"0.x;"       \
    NL "    fma.rn"type"    t0.x, "ma"0.y, "mb"1.x, t0.x;" \
    NL "    mul"type"       t0.y, "ma"0.x, "mb"0.y;"       \
    NL "    fma.rn"type"    t0.y, "ma"0.y, "mb"1.y, t0.y;" \
    NL "    mul"type"       t1.x, "ma"1.x, "mb"0.x;"       \
    NL "    fma.rn"type"    t1.x, "ma"1.y, "mb"1.x, t1.x;" \
    NL "    mul"type"       t1.y, "ma"1.x, "mb"0.y;"       \
    NL "    fma.rn"type"    t1.y, "ma"1.y, "mb"1.y, t1.y;" \
    NL "    mul"type"       t2.x, "ma"2.x, "mb"0.x;"       \
    NL "    fma.rn"type"    t2.x, "ma"2.y, "mb"1.x, t2.x;" \
    NL "    mul"type"       t2.y, "ma"2.x, "mb"0.y;"       \
    NL "    fma.rn"type"    t2.y, "ma"2.y, "mb"1.y, t2.y;" \
    NL "    mov.v2"type"    "dm"0, t0;"                    \
    NL "    mov.v2"type"    "dm"1, t1;"                    \
    NL "    }"

#define ptx_mul_m3m3(type, dm, ma, mb)                     \
    NL "    /* "dm" = "ma" * "mb"; */ {"                   \
    NL "    .reg .v4 "type" t<3>;"                         \
    NL "    mul"type"       t0.x, "ma"0.x, "mb"0.x;"       \
    NL "    fma.rn"type"    t0.x, "ma"0.y, "mb"1.x, t0.x;" \
    NL "    fma.rn"type"    t0.x, "ma"0.z, "mb"2.x, t0.x;" \
    NL "    mul"type"       t0.y, "ma"0.x, "mb"0.y;"       \
    NL "    fma.rn"type"    t0.y, "ma"0.y, "mb"1.y, t0.y;" \
    NL "    fma.rn"type"    t0.y, "ma"0.z, "mb"2.y, t0.y;" \
    NL "    mul"type"       t0.z, "ma"0.x, "mb"0.z;"       \
    NL "    fma.rn"type"    t0.z, "ma"0.y, "mb"1.z, t0.z;" \
    NL "    fma.rn"type"    t0.z, "ma"0.z, "mb"2.z, t0.z;" \
    NL "    mul"type"       t1.x, "ma"1.x, "mb"0.x;"       \
    NL "    fma.rn"type"    t1.x, "ma"1.y, "mb"1.x, t1.x;" \
    NL "    fma.rn"type"    t1.x, "ma"1.z, "mb"2.x, t1.x;" \
    NL "    mul"type"       t1.y, "ma"1.x, "mb"0.y;"       \
    NL "    fma.rn"type"    t1.y, "ma"1.y, "mb"1.y, t1.y;" \
    NL "    fma.rn"type"    t1.y, "ma"1.z, "mb"2.y, t1.y;" \
    NL "    mul"type"       t1.z, "ma"1.x, "mb"0.z;"       \
    NL "    fma.rn"type"    t1.z, "ma"1.y, "mb"1.z, t1.z;" \
    NL "    fma.rn"type"    t1.z, "ma"1.z, "mb"2.z, t1.z;" \
    NL "    mul"type"       t2.x, "ma"2.x, "mb"0.x;"       \
    NL "    fma.rn"type"    t2.x, "ma"2.y, "mb"1.x, t2.x;" \
    NL "    fma.rn"type"    t2.x, "ma"2.z, "mb"2.x, t2.x;" \
    NL "    mul"type"       t2.y, "ma"2.x, "mb"0.y;"       \
    NL "    fma.rn"type"    t2.y, "ma"2.y, "mb"1.y, t2.y;" \
    NL "    fma.rn"type"    t2.y, "ma"2.z, "mb"2.y, t2.y;" \
    NL "    mul"type"       t2.z, "ma"2.x, "mb"0.z;"       \
    NL "    fma.rn"type"    t2.z, "ma"2.y, "mb"1.z, t2.z;" \
    NL "    fma.rn"type"    t2.z, "ma"2.z, "mb"2.z, t2.z;" \
    NL "    mov.v4"type"    "dm"0, t0;"                    \
    NL "    mov.v4"type"    "dm"1, t1;"                    \
    NL "    mov.v4"type"    "dm"2, t2;"                    \
    NL "    }"

#define ptx_mul_m4m4(type, dm, ma, mb)                     \
    NL "    /* "dm" = "ma" * "mb"; */ {"                   \
    NL "    .reg .v4 "type" t<4>;"                         \
    NL "    mul"type"       t0.x, "ma"0.x, "mb"0.x;"       \
    NL "    fma.rn"type"    t0.x, "ma"0.y, "mb"1.x, t0.x;" \
    NL "    fma.rn"type"    t0.x, "ma"0.z, "mb"2.x, t0.x;" \
    NL "    fma.rn"type"    t0.x, "ma"0.w, "mb"3.x, t0.x;" \
    NL "    mul"type"       t0.y, "ma"0.x, "mb"0.y;"       \
    NL "    fma.rn"type"    t0.y, "ma"0.y, "mb"1.y, t0.y;" \
    NL "    fma.rn"type"    t0.y, "ma"0.z, "mb"2.y, t0.y;" \
    NL "    fma.rn"type"    t0.y, "ma"0.w, "mb"3.y, t0.y;" \
    NL "    mul"type"       t0.z, "ma"0.x, "mb"0.z;"       \
    NL "    fma.rn"type"    t0.z, "ma"0.y, "mb"1.z, t0.z;" \
    NL "    fma.rn"type"    t0.z, "ma"0.z, "mb"2.z, t0.z;" \
    NL "    fma.rn"type"    t0.z, "ma"0.w, "mb"3.z, t0.z;" \
    NL "    mul"type"       t0.w, "ma"0.x, "mb"0.w;"       \
    NL "    fma.rn"type"    t0.w, "ma"0.y, "mb"1.w, t0.w;" \
    NL "    fma.rn"type"    t0.w, "ma"0.z, "mb"2.w, t0.w;" \
    NL "    fma.rn"type"    t0.w, "ma"0.w, "mb"3.w, t0.w;" \
    NL "    mul"type"       t1.x, "ma"1.x, "mb"0.x;"       \
    NL "    fma.rn"type"    t1.x, "ma"1.y, "mb"1.x, t1.x;" \
    NL "    fma.rn"type"    t1.x, "ma"1.z, "mb"2.x, t1.x;" \
    NL "    fma.rn"type"    t1.x, "ma"1.w, "mb"3.x, t1.x;" \
    NL "    mul"type"       t1.y, "ma"1.x, "mb"0.y;"       \
    NL "    fma.rn"type"    t1.y, "ma"1.y, "mb"1.y, t1.y;" \
    NL "    fma.rn"type"    t1.y, "ma"1.z, "mb"2.y, t1.y;" \
    NL "    fma.rn"type"    t1.y, "ma"1.w, "mb"3.y, t1.y;" \
    NL "    mul"type"       t1.z, "ma"1.x, "mb"0.z;"       \
    NL "    fma.rn"type"    t1.z, "ma"1.y, "mb"1.z, t1.z;" \
    NL "    fma.rn"type"    t1.z, "ma"1.z, "mb"2.z, t1.z;" \
    NL "    fma.rn"type"    t1.z, "ma"1.w, "mb"3.z, t1.z;" \
    NL "    mul"type"       t1.w, "ma"1.x, "mb"0.w;"       \
    NL "    fma.rn"type"    t1.w, "ma"1.y, "mb"1.w, t1.w;" \
    NL "    fma.rn"type"    t1.w, "ma"1.z, "mb"2.w, t1.w;" \
    NL "    fma.rn"type"    t1.w, "ma"1.w, "mb"3.w, t1.w;" \
    NL "    mul"type"       t2.x, "ma"2.x, "mb"0.x;"       \
    NL "    fma.rn"type"    t2.x, "ma"2.y, "mb"1.x, t2.x;" \
    NL "    fma.rn"type"    t2.x, "ma"2.z, "mb"2.x, t2.x;" \
    NL "    fma.rn"type"    t2.x, "ma"2.w, "mb"3.x, t2.x;" \
    NL "    mul"type"       t2.y, "ma"2.x, "mb"0.y;"       \
    NL "    fma.rn"type"    t2.y, "ma"2.y, "mb"1.y, t2.y;" \
    NL "    fma.rn"type"    t2.y, "ma"2.z, "mb"2.y, t2.y;" \
    NL "    fma.rn"type"    t2.y, "ma"2.w, "mb"3.y, t2.y;" \
    NL "    mul"type"       t2.z, "ma"2.x, "mb"0.z;"       \
    NL "    fma.rn"type"    t2.z, "ma"2.y, "mb"1.z, t2.z;" \
    NL "    fma.rn"type"    t2.z, "ma"2.z, "mb"2.z, t2.z;" \
    NL "    fma.rn"type"    t2.z, "ma"2.w, "mb"3.z, t2.z;" \
    NL "    mul"type"       t2.w, "ma"2.x, "mb"0.w;"       \
    NL "    fma.rn"type"    t2.w, "ma"2.y, "mb"1.w, t2.w;" \
    NL "    fma.rn"type"    t2.w, "ma"2.z, "mb"2.w, t2.w;" \
    NL "    fma.rn"type"    t2.w, "ma"2.w, "mb"3.w, t2.w;" \
    NL "    mul"type"       t3.x, "ma"3.x, "mb"0.x;"       \
    NL "    fma.rn"type"    t3.x, "ma"3.y, "mb"1.x, t3.x;" \
    NL "    fma.rn"type"    t3.x, "ma"3.z, "mb"2.x, t3.x;" \
    NL "    fma.rn"type"    t3.x, "ma"3.w, "mb"3.x, t3.x;" \
    NL "    mul"type"       t3.y, "ma"3.x, "mb"0.y;"       \
    NL "    fma.rn"type"    t3.y, "ma"3.y, "mb"1.y, t3.y;" \
    NL "    fma.rn"type"    t3.y, "ma"3.z, "mb"2.y, t3.y;" \
    NL "    fma.rn"type"    t3.y, "ma"3.w, "mb"3.y, t3.y;" \
    NL "    mul"type"       t3.z, "ma"3.x, "mb"0.z;"       \
    NL "    fma.rn"type"    t3.z, "ma"3.y, "mb"1.z, t3.z;" \
    NL "    fma.rn"type"    t3.z, "ma"3.z, "mb"2.z, t3.z;" \
    NL "    fma.rn"type"    t3.z, "ma"3.w, "mb"3.z, t3.z;" \
    NL "    mul"type"       t3.w, "ma"3.x, "mb"0.w;"       \
    NL "    fma.rn"type"    t3.w, "ma"3.y, "mb"1.w, t3.w;" \
    NL "    fma.rn"type"    t3.w, "ma"3.z, "mb"2.w, t3.w;" \
    NL "    fma.rn"type"    t3.w, "ma"3.w, "mb"3.w, t3.w;" \
    NL "    mov.v4"type"    "dm"0, t0;"                    \
    NL "    mov.v4"type"    "dm"1, t1;"                    \
    NL "    mov.v4"type"    "dm"2, t2;"                    \
    NL "    mov.v4"type"    "dm"3, t3;"                    \
    NL "    }"

// mul_mX_f32
#define mul_m2_f32(dm, ma, mb) ptx_mul_m2m2(".f32", dm, ma, mb)
#define mul_m3_f32(dm, ma, mb) ptx_mul_m3m3(".f32", dm, ma, mb)
#define mul_m4_f32(dm, ma, mb) ptx_mul_m4m4(".f32", dm, ma, mb)

// generic vector length operations
#define ptx_len_v2(type, d, v)                  \
    NL "    // "d" = length("v");"              \
            ptx_dot_v2          (type, d, v, v) \
    NL "    sqrt.approx"type"   "d", "d";"

#define ptx_len_v3(type, d, v)                  \
    NL "    // "d" = length("v");"              \
            ptx_dot_v3          (type, d, v, v) \
    NL "    sqrt.approx"type"   "d", "d";"

#define ptx_len_v4(type, d, v)                  \
    NL "    // "d" = length("v");"              \
            ptx_dot_v4          (type, d, v, v) \
    NL "    sqrt.approx"type"   "d", "d";"

// len_vX_f32
#define len_v2_f32(d, v) ptx_len_v2(".f32", d, v)
#define len_v3_f32(d, v) ptx_len_v3(".f32", d, v)
#define len_v4_f32(d, v) ptx_len_v4(".f32", d, v)

// generic vector normalization operations
#define ptx_norm_v2(type, d, v)                        \
    NL "    /* "d" = normalize("v"); */ {"             \
    NL "    .reg "type"         t;"                    \
            ptx_dot_v2          (type, "t", v, v)      \
    NL "    rsqrt.approx"type"  t, t;"                 \
            ptx_op_v2_s         ("mul"type, d, v, "t") \
    NL "    }"

#define ptx_norm_v3(type, d, v)                        \
    NL "    /* "d" = normalize("v"); */ {"             \
    NL "    .reg "type"         t;"                    \
            ptx_dot_v3          (type, "t", v, v)      \
    NL "    rsqrt.approx"type"  t, t;"                 \
            ptx_op_v3_s         ("mul"type, d, v, "t") \
    NL "    }"

#define ptx_norm_v4(type, d, v)                        \
    NL "    /* "d" = normalize("v"); */ {"             \
    NL "    .reg "type"         t;"                    \
            ptx_dot_v4          (type, "t", v, v)      \
    NL "    rsqrt.approx"type"  t, t;"                 \
            ptx_op_v4_s         ("mul"type, d, v, "t") \
    NL "    }"

// norm_vX_f32
#define norm_v2_f32(d, v) ptx_norm_v2(".f32", d, v)
#define norm_v3_f32(d, v) ptx_norm_v3(".f32", d, v)
#define norm_v4_f32(d, v) ptx_norm_v4(".f32", d, v)

// neg_vX_f32
#define neg_v2_f32(d, v) ptx_op_v2("neg.f32", d, v)
#define neg_v3_f32(d, v) ptx_op_v3("neg.f32", d, v)
#define neg_v4_f32(d, v) ptx_op_v4("neg.f32", d, v)

// abs_vX_f32
#define abs_v2_f32(d, v) ptx_op_v2("abs.f32", d, v)
#define abs_v3_f32(d, v) ptx_op_v3("abs.f32", d, v)
#define abs_v4_f32(d, v) ptx_op_v4("abs.f32", d, v)

// min_vX_f32
#define min_v2_f32(d, v) ptx_op_v2("min.f32", d, v)
#define min_v3_f32(d, v) ptx_op_v3("min.f32", d, v)
#define min_v4_f32(d, v) ptx_op_v4("min.f32", d, v)

// max_vX_f32
#define max_v2_f32(d, v) ptx_op_v2("max.f32", d, v)
#define max_v3_f32(d, v) ptx_op_v3("max.f32", d, v)
#define max_v4_f32(d, v) ptx_op_v4("max.f32", d, v)

#endif
